{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from attention_visualization import *\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "# 为了支持中文显示\n",
    "from matplotlib.font_manager import FontProperties\n",
    "font = FontProperties(fname='./SimHei.ttf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Pretrainer(vocab_size=config[\"vocab_size\"], #字数量\n",
    "                   max_seq_len=256, # 最大句长\n",
    "                   batch_size=1, # batch_size只能为1\n",
    "                   )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = \"为什么要上班\"\n",
    "attention_matrices = model(text)\n",
    "print(attention_matrices[0].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgEAAAHcCAYAAABPrnkNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xm0ZGV57/Hv0+f0ALSAyNDNICAqUVTUBY6goteoV0KcEzRXRBCN4kREjYrBIXqjGHGWNiIhCVGXLtdVo0LECQiiDaIgCAg0zdBMKtBAT3Q/9493l10Wp/uc7j5Vu+q8389atWrXHp/qPmfXr9733ftEZiJJkuozq+0CJElSOwwBkiRVyhAgSVKlDAGSJFXKECBJUqUMAZIkVcoQIElSpQwBkiRVyhAgSVKlxtsuYNjMmb1Nzpu3fdtlTLtYO3PvDJnRdgV9EjPzja2bM3O/e4zdu6btEvpjhv4sAjBD75p716pbbs/MnSZbzxDQY9687Tnwca9vu4xpN3736rZL6Jscn5kfKjlrZp54795rm7ZL6JvtLrql7RL6Y2ys7Qr6Jtbc13YJffG9q0+6birrzcyzpyRJmpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkSk0pBETEwyNiRURE8/qbEfHWnnUeGhHfjoi7I+K2iPhkRMxrlu0VERkRx05ynIiI10fEbyJiZURcGRFHTrDen0fEFV2vfxURL5zKe5EkScVUWwIeBlydmdn1+qrOwojYFTgX2Bt4OXAc8ArgXzexnvcDJwGfAg4FfgacGhFPn6Ceq5pjB/DQ7nokSdLkxje2MCLmArOBRwJLImI+JTg8BFgWEXMyczVwArAj8NTMvLrZdlvg0xFxArB6ivW8CfhSZn6m2ccPgUOA1wM/joitgDHgEcDSpp6FwFbALRExnpn3Tf3tS5JUr42GAOAU4Iiu18u7phcD74uI9wEvBc7vBIDGT5vnA4HzJiskIsYoH+Y7duZl5tqmmX9dM+u7QHerwN92Td8KHAmcNtmxJEnS5N0B/wgcDJxDaaI/mPKt/A/N9KnAHsCDgF/3bHs58BeUboJJZeZa4BvAyyLiXyJin2b+zzJzcbPaG5vjXgW8q5n+AHBZM/2dqRxLkiRNEgIy86rMPBeYC5zXTN8OLMnMczNzKbCgWf13Pdvem5nfzszrNqGeo4HTgVcDV0bEVzphoNnnJU0N2wE/aKaXA1c09dy6CceSJKlqUx0YuDtwfdf00q5lc5rntVtaTGYuz8wjKGMQTgNeCFwYEQd11omIOcBOG6lnk0XEMRGxOCIWr15zz5bsSpKkkbHREBARs5rL/BZQBgKOAw8GboiI8YiYxfpxAvN7tt0tIs6NiMM2tajM/E1mHgU8hTIe4PRmn2PAnpTAcfsE9cSmHqs53qLMPCAzD5gze5vN2YUkSSNnspaAU4EVzXrXAGuAtwBvaKbfC/yW8kH9sJ5tdwWeyvqWgo2KiEMj4tKI2LczrxkL8Elg74jYGTgbuJIyoHFVU8MLgI8200fcb8eSJGlCk4WAEymD8W6gjPI/kPIhfHwzvSgz7wF+CBwSETt1bXsIkJRr/afiNmA/SnDotoASRO4AXgt8GPh5Vz3LKfckOBD41hSPJUlS9TZ6iWBmLomIdcBlnRH6EbEQODMzL+la9e2UqwC+HREnAvsA/wCckZlLI2KvZr39IuLQCQ51NuWD/RzgpOZ+AJdTAsFRwMnN/QiuaLoAfpGZiyNiB+ABwLcyc/kE+5UkSRsw2X0CAPalfPvvBID59NydLzMvioiDgY8AXwfuBj4HvLtnX69rHr32bgLHYcAHgbcBO1O6IN7a7Ku7np90Td9sAJAkadNNGgIy881d08vYQBdCZl4IPGsDy5YAkw7ay8w7gGObx4bW+cuu6fMpdwyUJEmbyL8iKElSpQwBkiRVyhAgSVKlDAGSJFXKECBJUqUMAZIkVcoQIElSpQwBkiRVyhAgSVKlDAGSJFXKECBJUqUMAZIkVcoQIElSpQwBkiRVyhAgSVKlDAGSJFXKECBJUqUMAZIkVcoQIElSpcbbLmDYxIrVzP71dW2XMe3W3nlX2yX0zZpnPrbtEvrinZ8/ve0S+uLkJx7cdgl9s/Z3v2+7hP6IaLuCvomxsbZLaJUtAZIkVcoQIElSpQwBkiRVyhAgSVKlDAGSJFXKECBJUqUMAZIkVcoQIElSpQwBkiRVyhAgSVKlDAGSJFXKECBJUqUMAZIkVcoQIElSpQwBkiRVyhAgSVKlDAGSJFXKECBJUqUMAZIkVcoQIElSpYY6BETE/hEx1kwfFhFLIuLYtuuSJGkmGNoQEBFPBC4GDm9m7QTsCVzXWlGSJM0gQxsCMvMC4OfAO5pZOzTPv22nIkmSZpahDQGNjwK/ioitgQc1865vsR5JkmaM8bYL6BURjwTWAmsoLQE/B3YG9gBWAjtGxI49m63IzFsGWqgkSSNu6EIAcBaw20aWXzvBvK8DL+lPOZIkzUzD2B2wLzAXGMvMyMwAxoBbgZM787qW3QIsaa9cSZJG09C1BGTmPRPMPpjSJfCd7pnN5YM7AUu35JgRcQxwDMC8WfO3ZFeSJI2MYWwJmMgbgBuBH0TE9l3zd6a8hxu3ZOeZuSgzD8jMA+bEvC3ZlSRJI2PoQ0BE7A+8CPgscBpwVkR06l7YPG9RCJAkqUZDHQIiYi7wBeA24FPAIuAA4DXNKgua55sGX50kSaNt6MYEdETEbOB04EDgsMxcDpwTEd9g/T0DFgAJLGunSkmSRtdQhoCIeCDlsr9DgBMy81tdi1+SmdlM7wrclplrBl2jJEmjbui6AyLiMMoNgg4BPpiZH+xe3gkATUvBoTgeQJKkzTJUISAiTgT+H+XugEdn5gkTrHNKRFwH/B54InD+QIuUJGmGGLbugA9R7hb46cz85QbW+QHwMuBq4DzgPQOqTZKkGWWoQkBmrmb9yP8NrfMV4CuDqUiSpJlrqLoDJEnS4BgCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEqNt13A0MkkV61qu4ppF7Nn7n/1vJ9d1XYJfXH6rU9tu4S+uPll+7ZdQt/s9Pmftl1CX8ScOW2X0D/rsu0KWmVLgCRJlTIESJJUKUOAJEmVMgRIklQpQ4AkSZUyBEiSVClDgCRJlTIESJJUKUOAJEmVMgRIklQpQ4AkSZUyBEiSVClDgCRJlTIESJJUKUOAJEmVMgRIklQpQ4AkSZUyBEiSVClDgCRJlRraEBARz42IJ0wwf9eIODYiHt5GXZIkzRRDGwKAjwJnRcTCnvl7AJ8C/mzwJUmSNHOMt13ARCLiqcCjgLdm5rKexfc2z3cMtipJkmaWYW0JeCewHPjiBMuyeV43uHIkSZp5hi4ENOMADgUuy8zlbdcjSdJMNVQhICIC+FjPvAdExNxJthuPiG37WpwkSTPMsI0JOBo4CLika945wP4lH/yJc3rm3Qls39fqJEmaQYYmBETEQ4CTga8AN1DCAMArKHWual4/DPhmM/+iZt44sNHWgkmOfQxwDMC82GZzdyNJ0kgZmhAAXAt8GXgXcHxnZmb+unuliOjUvDQzfzMdB87MRcAigO3GdsxJVpckaUYYmhCQmQkcBTBB078kSZpmQzUwUJIkDY4hQJKkShkCJEmqlCFAkqRKDWsIiOYhSZL6ZFhDwDgwtoFl85rnOQOqRZKkGWlYQ8BWwNYbWGYIkCRpGgzNfQK6ZeYxG1l2LnYVSJK0xYa1JUCSJPWZIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkSo23XcCwWbXLVlx31GPbLmPa7fGh89suoW/WrV3bdgl9cdtB2XYJfbFw16Vtl9A39+XM/D/L1avbLqFvVp25Z9sl9Mezp7aaLQGSJFXKECBJUqUMAZIkVcoQIElSpQwBkiRVyhAgSVKlDAGSJFXKECBJUqUMAZIkVcoQIElSpQwBkiRVyhAgSVKlDAGSJFXKECBJUqUMAZIkVcoQIElSpQwBkiRVyhAgSVKlDAGSJFVqZEJAROzcdg2SJM0kIxECImIH4MqIOCUi5rRdjyRJM8FIhADgOGA74LzMXN12MZIkzQRDHwIiYlfgjcBXMvP0Zt54u1VJkjT6hj4EAJ8B1gJvAoiIAM6JiPdFxNxWK5MkaYQNdQiIiJcALwDelpm3NrP/GngS8DRgrK3aJEkadUMbAiJiT+AU4AeZeWozbz7wT8C1wIsy894WS5QkaaQNZd96RGwHfBNI4FVdi04EFgIHZeYfBl+ZJEkzx9CFgIhYSAkAfwY8JzOvb+YfBLwFeHdmXjDNxzwGOAZgfNsHTueuJUkaWsPYHfBm4HHA/8nMHwFExPbAvwFnAh+Z7gNm5qLMPCAzDxjbZpvp3r0kSUNp6EJAZr4TeGxmfhUgImYBXwbuA16RmRkRsyPi6Ih4epu1SpI0yoauOwAgMy/tenky5WqAp2TmHc28XYAPASsj4jFd8yVJ0hQNXUtAt4h4P/A64MWZeVlnfmbeALwa2IMSEiRJ0iYayhAQEXMj4tPA3wOHZ+bZzfyxiNguInYHrgZ+BBwREc9vr1pJkkbTUHYHAD8HHg0sBz4QEZ8Atgc2NGpvUUTsZ7eAJElTN6wh4CxKALgGuB64CbgVuK15vhO4u3m8A3gf5c6Cp7VQqyRJI2koQ0Bmvm2q60bEPwPfzcyL+liSJEkzzlCGgE2RmSsAA4AkSZtoKAcGSpKk/jMESJJUKUOAJEmVMgRIklQpQ4AkSZUyBEiSVClDgCRJlTIESJJUKUOAJEmVMgRIklQpQ4AkSZUyBEiSVClDgCRJlTIESJJUKUOAJEmVMgRIklSp8bYLGDaRMOu+tquQYNac2W2X0Bdrd9q+7RL6ZnyG/p/94QkL2i6hb2adkm2X0CpbAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkSo23XUC3iNgKWAeszsyc4jazgDlAZuaqftYnSdJMMmwtAf8JrATWRURO5QGsBVYAf99q5ZIkjZihagkAjgPeTflQX9fMexewL3BE8/p84GPA15rXY8BWwO8GV6YkSaNvqEJAZl7TOy8i7gJWZOaS5vVa4PbOa0mStHmGKgQARMSzgO9PML97jMCXIuJLXa+fnJk/7XtxkiTNIMM2JgBKVwDA/sDewBeAnzTTewM3A8c3009o1l054BolSRp5Q9cSwPqxAEsz846NdQdEROfDf0pXEkiSpPWGMQRs3Tz/ISL+OHOS7oB5W3LAiDgGOAZgfNsHbsmuJEkaGcMYAv6H0tTf7W+AS4GLN7DNsi05YGYuAhYBbLVwD1sVJElVGJoQEBFjwGxgVffI/4jYA3gPcCFwSGaunmDb+RGxM3BjZq7rXS5Jku5vaEIAcDDwQ4DuboAuTwFWbWBZxwOBO6a9MkmSZqBhujrgAuAhwEJgp+bxeeBO4Eld8x4N3EO5sVBn3u7Aw4C7Bl61JEkjamhaAjJzBXBt53VEnAC8Dvg65Y6AKym3CP405ZLBk6f69wUkSdL9DU0ImMDXgBuBJ1M++PcFfg9sDzzbACBJ0pYZ2hCQmZcDl0fEeZS/G/AGyhUC48API2IZ8B1KS8HZmXlfa8VKkjSChioERMTzgedRrvtfQOn/3x04Gzg8M7/TrLcL8ELgSOA1wO8i4pWd5ZIkaXJDFQKAxcDHgSXAFcBXgTMz85bulZrXnwc+HxGPA54LnDnYUiVJGm1DFQKaD/eHb+I2vwB+0Z+KJEmauYbpEkFJkjRAhgBJkiplCJAkqVKGAEmSKmUIkCSpUoYASZIqZQiQJKlShgBJkiplCJAkqVKGAEmSKmUIkCSpUoYASZIqZQiQJKlShgBJkiplCJAkqVKGAEmSKjXedgHDZvbN97DbRy9ouwxtgrzvvrZL6IuZ+r5mXbmk7RL65oaj92+7hL54wst/2XYJfXPj/57bdgmtsiVAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKjUQIiIjXRsSlEbFL27VIkjRTjEQIAB4A7Afc13YhkiTNFKMSAlY2z6tarUKSpBlkVEKAJEmaZuNtF7CJtomI7tezgXnAXZl5TzslSZI0mkatJeBmYHnX4/fATcBL2yxKkqRRNGotAQuA7m/8s4A5PfMkSdIUjFoIuCcz757unUbEMcAxAPPYerp3L0nSUBq17oC+yMxFmXlAZh4wm7ltlyNJ0kAYAiRJqpQhQJKkShkCJEmqlCFAkqRKDdXVARFxOOUGQKt7Fj2+eX5xREx06+AxgMz8jz6WJ0nSjDJUIQD4MLAjE/+hoDuBT2xgu3HgbsAQIEnSFA1VCMjMvdquQZKkWjgmQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqlRkZts1DJVtt9k1n7Tfa9suY9rl4kvbLqF/ItquoD9m6O9mzJ7Tdgl9E2N+rxo1373mp22X0BdjC397YWYeMNl6/sRKklQpQ4AkSZUyBEiSVClDgCRJlTIESJJUKUOAJEmVMgRIklQpQ4AkSZUyBEiSVClDgCRJlTIESJJUKUOAJEmVMgRIklQpQ4AkSZUyBEiSVClDgCRJlTIESJJUKUOAJEmVMgRIklQpQ4AkSZUaqRAQETtGxHjbdUiSNBMMVQiIiPkRMafr9VYRsXXXKhcBZ0bE1hExJyKiWW+82TYGXbMkSaNqqEIA8CNgVURkRCRwL/DJruX3AdcApwOrgHXNemuA5cAugy1XkqTRNWwh4IXAnsANwNuABwPv6lq+BrgLOBbYHVgGvAXYFXgIcNsgi5UkaZQNVQjIzOszcymwFrgzM68HxiPidV3dBPdm5s2ZeSOwrllvWWZem5lr26pdkqRRM1QhYAMeDHwOiOaxot1yJEmaGUYhBKxpnldTQsDKFmuRJGnGGKrL7SLiUcAcYDawe0Q8CRgDyMyMiG2BFRGxW7POGLBDROwDrG66DzbnuMcAxwDMm7Pdlr8RSZJGwLC1BHwc+CmwkDIg8Dxgb4CImA3sQLli4BPAtcAC4GPAbyldBpslMxdl5gGZecDs8a0n30CSpBlgqEJAZj47M+cAS4HXZ+ZYMw0lGIwDNwJHAvOb6aOBbYCXD75iSZJG11B1B2zA+cBOwAHN66szczlAc2+gtZl5b0u1SZI0soaqJWAimbkmM28HngYsy8xr265JkqSZYOhbAiJiHuUKgZcA/91yOZIkzRhDFwKaKwBmA6+KiDcB3wcuBB4GHNVmbZIkzSRDEwIiYk/g28AjKfcDWAZ8l/L3BE4BfgZcEBGPp9wrYB2l/oURsR+wFXBVZt45+OolSRo9QxMCMvO6iLgNeAdwRmbeBBARP6ZcGfACyt8IuJBymWDnJkLvAP4emAc8Dzh7wKVLkjSShiYEAGTmMyeYfQTwjMy8KCJmAVtlpncNlCRpCw1VCJhIZi4BTmum1+FtgyVJmhZDf4mgJEnqD0OAJEmVMgRIklQpQ4AkSZUyBEiSVClDgCRJlTIESJJUKUOAJEmVMgRIklQpQ4AkSZUyBEiSVClDgCRJlTIESJJUKUOAJEmVMgRIklQpQ4AkSZUab7uAYRP3rWXsljvaLmPa5cP3abuEvokVq9ouQZtiVrRdQd+s2XWHtkvoi7HlM/d37OGnP67tEvrk76a0li0BkiRVyhAgSVKlDAGSJFXKECBJUqUMAZIkVcoQIElSpQwBkiRVyhAgSVKlDAGSJFXKECBJUqUMAZIkVcoQIElSpQwBkiRVyhAgSVKlDAGSJFXKECBJUqUMAZIkVcoQIElSpQwBkiRVyhAgSVKlphQCIuLhEbEiIqJ5/c2IeGvPOn8VERc36y2JiHf2LM8NPH7ULN+rZ/6aiLgiIk6MiLk9+/rziLii6/WvIuKFm/lvIElSlcanuN7DgKszM7teL+osjIijmtcnABcAhwIfjohlmfmvXfs5A/hUz77v6nl9CnAasA3wRODdwG7Aa3rquao5dgAP7byWJElTs9EQ0HwDnw08ElgSEfMprQcPAZZFxJzMXA28GfheZn6o2fTsiHgK8HqgOwQsy8yfTlLTDV3rnN3U8N6IeDuwEhgDHgEsbepZCGwF3BIR45l539TeuiRJdZusJeAU4Iiu18u7phcD7wNOpHxrf1DPtkcD229hfQCXNs/7ACcBT+9a9rdd07cCR1JaESRJ0iQmGxPwj8DBwDmUZvyDKd/u/9BMn9qs9zXgiRHxjYjYHyAzL8nMc3r2FxEx3vOISWrYtXm+EXhjc9yrgHc10x8ALmumvzPJviRJUmOjISAzr8rMc4G5wHnN9O3Aksw8NzOXNqu+B/gY8Hzg4og4MyIeN8EujwPW9Dxe3FtTEw62jYhnAccD38/MZU2wOBfYDvhBM70cuKKp59bN+DeQJKlKU71EcHfg+q7ppd0LM3NNZr6N0mT/ceAg4PyIeEnPfv4dOLDncXbPOu+jhIM7ge8DFwOv7CyMiDnAThurZ1NFxDERsTgiFq9eu2JLdiVJ0siYbGDgLGAOsIAyEHAceDBwQzO9LjPXddbPzOuB4yLis8BZwBcj4nuZeXezyi2ZuXiSmj5H6WZYA1yTmX8chxARY8CewFrg9q56zmum13ZdwTBlmbmI5mqH7ebussnbS5I0iiZrCTgVWNGsdw3lg/ktwBua6fdGxOMj4tKIeGZno8z8LfB+YFvKlQWb4qbMXJyZv+wOAI2zgSsp4WVVU8MLgI8200cgSZKmZLIQcCJlMN4NrG++v5LST38g5dvzjcB+wDN6tl3QPC+bnlIBeC3wYeDnXfUsB17RTH9rGo8lSdKMttHugMxcEhHrgMs6zfgRsRA4MzMv6awXEWcAx0fEvZSbBT2GMnr/a00XQcfCiHjSBIe6aCrFZuYVTbP/LzJzcUTsADwA+NYErQaSJGkjpnLHwH0p3/47AWA+978731GU7oJXA++lDNo7CfhIz3ovbx699ph6yewL/KRr+mYDgCRJm27SEJCZb+6aXsYEXQiZuZJyy+ATNrKfye4HADDpOpn5l13T51PuGChJkjaRf0VQkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSaqUIUCSpEoZAiRJqpQhQJKkShkCJEmqlCFAkqRKGQIkSapUZGbbNQyViLgNuG5Ah9sRuH1Axxq0mfrefF+jZ6a+N9/X6Bnke9szM3eabCVDQIsiYnFmHtB2Hf0wU9+b72v0zNT35vsaPcP43uwOkCSpUoYASZIqZQho16K2C+ijmfrefF+jZ6a+N9/X6Bm69+aYAEmSKmVLgCQNsYh4bURcGhG7tF2LZh5DgARExHMj4gkTzN81Io6NiIe3UVc/RMTObdewuSJiq4iYGxGxCdvMioh5ETG3n7X10QOA/YD72i6kHyJix4gYb7uOWhkCNiAiHh4RKzonm4j4ZkS8tWedh0bEtyPi7oi4LSI+GRHzmmV7RURGxLGTHCci4vUR8ZuIWBkRV0bEkROs9+cRcUXX619FxAun591umojYPyLGmunDImLJZO9zBHwUOCsiFvbM3wP4FPBngy9p+kXEDsCVEXFKRMxpu57N8J/ASmBd8/s16QNYC6wA/r7VyjffyuZ5VatVbKaImN/9s9YEua27VrkIODMito6IOV3n3PFm2ykHvi2scyrn/L+KiIub9ZZExDt7lm/o5/BHzfK9euaviYgrIuLE3pA6qHO+IWDDHgZcnesHTTwMuKqzMCJ2Bc4F9gZeDhwHvAL41008zvuBkygfNIcCPwNOjYinT1DPVc2xA3hodz2DEhFPBC4GDm9m7QTsyeBusDTtIuKpwKOAEzNzWc/ie5vnOwZbVd8cB2wHnJeZq9suZjMcR/m/2ofyu7c38AXgJ12vbwaO73r9UODRwCkt1Cv4EbCqK5TdC3yya/l9wDXA6ZSgs65Zbw2wHBhUN8hk5/yjgDOAr1LO1d8APhwRR/Ts5wzgyT2P1/esc0oz/7mUz4zjgU9PUE/fz/k2wfRo0ths4JHAkoiYTwlLDwGWRcSc5uR5AuXuT0/NzKubbbcFPh0RJwBTPcG+CfhSZn6m2ccPgUMoPzQ/joitgDHgEcDSpp6FwFbALRExnpkDaybMzAsi4ufAO4B/B3ZoFv12UDX0wTspJ5svTrCsc0JYN7hy+qMJrm8EvpKZpzfzBvrzs6Uy85reeRFxF7AiM5c0r9cCt3dezyDb9Hwpng3MA+7KzHvaKWlKXggEcB5wMuVDtLtVYw1wF+Wc+mbg58A/NevNA27rZ3GbcM5/M/C9zPxQs+nZEfEUyrm6+8vfssz86SSHvaFrnbObGt4bEW+ntPwM7JxvS8D9nUL5QPgI8Pxm+k5gDrAYeFeTyl4KnN8JAI3Of+qBUzlQ06S+FSVMAJCZaym/NB9tZn23qeENwN8201c2y24F/mbT3t60+Cjwq6ZJ70HNvOtbqGOLRRkHcChwWWYub7uePvsMpWn8TfDHbxfnRMT7epsih1lEPKunuf/vgOd0vd4N+FJPs+uT2q16WtyUyxvTAAAJqUlEQVRM+f3vPH4P3EQ5Fw2tzLw+M5dSfvbuzMzrgfGIeF1XN8G9mXlzZt5ICdx3ZuayzLy2OSf206Tn/Ga9bVh/vus4GnjbNNRwafO8DwM+5xsC7u8fgYOBcyhN9AdTkt4fmulTKf3EDwJ+3bPt5cBfULoJJtX8cH8DeFlE/EtE7NPM/1lmLm5We2Nz3KsoP4wHAx8ALmumv7NZ73ITRcQjI2LfiHgIJam/G9iZ8m+xEtix6e/qfgz1aObmQ/BjPfMeMNkHYtNXuW1fi5tmEfES4AXA2zLz1mb2XwNPAp5G+eYxKlY0z/szeXdAZ7DnSkbfAsogwc5jO0p33FfaLGozPRj4HKWFIFj/f9qGqZzzAb4GPDEivhER+wNk5iWZeU7P/qI5R3Q/JhvXsGvzfCMDPufbHdAjM68Crmo+CM7LzHOjDBZbkpnnwh+/PQL8rmfbe4FvN+vsNcVDHk05Qb0aODIivga8q9PCkJmXNPvbDvhB0xz/ZOCKTj0DchblG9aGXDvBvK8DL+lPOdPiaOAg4JKueecA+0/wO3tOz7w7ge37Wt00iYg9Kd92fpCZpzbz5lOaXK8FXtT87I6KTtfM0sy8Y2PdARHR+fCfCTdEuScz7267iGmypnleTQkBrYW0qZzzG++hhOU3AS+IiLOAd2bmL3p2eVzz6PZSSojomBXlioitKS3HxwPfb8YkLYPBnfNtCdiw3VnfxL07sLRrWacJa4ubqTJzeWYeQemPOo3SFXBhRBzUWadpMttpI/UMwr7AXGAsMyMzg/ILcStwcmde17JbgCUDrnHKmhaNkynfos7qWvQK4LGU/rhHAId1ze/MezTwrIEVuwWaE8k3KR+Cr+padCKln/HwzPzD4CvbIp2R5X+YrDuA5oRK6VtWCyLiURHxeEq/++5N18w8gGYQ3rbAiojYrfnyNAbsEBH7RMQeAyx1Y+d8MnNNZr6N0mT/ccoXiPObVrZu/075YO9+nN2zzvsoQehO4PuUwdav7Cwc5DnfloAeETGL8iG/gDIoZJzSdHVDM72O0kcDML9n290oHyofAX61KcfNzN8AR0XE5ygfSqcDD2nGDexJCRy3d9VzXjO9tms0a99sYODRwZQugT9pnmpq3onBB5VNcS3wZUpz2/GdmZn5J108sf765aXN/9HIaL7NfJNyeeNzmr5YmoD5FuDdmXlBiyVurv+hNPV3+xtKv+rFG9im96oPDc7HgadTPm/eRRkAeARARMymDC6+F/gE8OJmm481j/+ijNnpm6mc8zPzjwODm9+j4yLis5Rz9Rcj4ntdrTS3dHXnbsjnKN0Ma4BruscjDfycn5k+uh6Ub+O5kceJlAEia4Fv9Wx7YLPOS4C9muljN3KsQyknrn175p/YbLsz5fKajdXzqhb/rb4K3EBJ7tt3zV/Y1Pbitv8/p/g+TgJ+uoFlj2rey0Ft17kZ7+v/Ui6/elnXvO0pAei/aG4bPiqP5udsXm/drB+Xch4wZwPbzm/Wm9X2+9iM931s8zM4v+1atvB9LAGObqaf1rynBzfP/4syzmGb5pxyFKXFZ9sB1DWVc/7jm3P1M3u2fVWzzhOa1wmctJFjdT4X3rORdQZ6zrc74P5OpAzMuIH1TTlXUr4tHggsyvKt+IfAIRGxU9e2h1D+k342xWPdRrkT2FN75i+gDJS5A3gt8GHKYLxOPcspzdMHAt/apHc3TZqBMS8CPkv5JTqrSdRQQgCUQS5qSWa+E3hsZn4V/viN58uUYPCKzMyImB0RR8f970sxjA6m/F78yY2CKC1Oc4Gn0HU9es86y5v1RmpA5wx2PqW18JHN66uzdI12WhzXZua9mXnXAGo5kUnO+ZRz2X7AM3q2XdA8T2dL00DP+XYH9MgymGgd5ZKxxfDHZtUzsxmk13g75SqAb0fEiZR+on8AzsjMpV0DA/eLiImas86m/CefA5wU5X4Al1MCwVGUfvbVwBVNE9AvMnNxlDu+PYDSCtHKJW3NAJovUELMpyh96D8GXkMZgNb5xbipjfq0XmZe2vXyZMrVAE/JzM7Nj3YBPgSsjIjHdM0fRhdQrt1ewfpb6H6AcuOq5wCdy3UXUC7XPQH4t2beXMrluIP4UNEkMnMNpan7aZTr6icaWDyoWqZ0zo+IM4DjI+Jeys/iYyjdG1/LpqutsXADl6ReNMV6BnrONwRMbF+a6zKbH4b59NypKTMvioiDKf3/XwfupvTzvLtnX69rHr32bn74DgM+SLnWdGfKnbPe2uyru56fdE3f3GIAmE0Zr3AgcFhTxzkR8Q3WX0O7gNIiYj/skIiI91N+Dp+XmZd15mfmDRHxasq3i5P508GDQyUzV9B1FUqUm3K9jvL7txWlS2At5c5rP6EE6ZG5KiAiDqcMnuu90djjm+cXR8REtw4eA8jM/+hjedMqyu3V11C6Tv+75XJgCud8ypezayhXcr2XMmjvJMpnQLeXN49emzLIcWDnfP+UsKYsIh5IOeEeApyQmR/sWhadE25EvAd4Y2YO9X0COiLiJEqf//3Se0Q8inIJ4cE52Esyp0XTavMxShPjX2fm15v5Y5QTXeea889SmjoPzcz/aqfaTRMRj+BPb826L+UGOtsDz87Mn2xk86ETEUsoNw7b1LvBjQN3Z+aCSddsSXNfjcspIW5byoj4Cykj6Z+WXdfaR8QNlD7z01ootTq2BGhKmhaLf6Z0e3ywOwDAHy/16bQUHMpojQfo3LBkJvo55ZLG5cAHIuITlA/JbTaw/qKI2G/IuwUAyMzLgcsj4jzKaPM3UAZvjQM/jIhllCtXvg6cnUN+e+TM3KvtGqZbc4+Kb1P6/oPSOvhdyuC3Uyjjpy5oLiFcSbn6apzSpL4fpYXnqsy8c/DV18EQoEk1Yx7+gdJMeXRm3u8e+xFxCuWPYexA+Yb52UHWuIXG2fAd8zrXl4/iX9yDcgnTckoz5vWUcRq3UsZz3Eq5Tvnu5vEOyvXLL6AM9hw6EfF84HmU/5cFlICzO2WMzeGZ+Z1mvV0o99w4kjJW5XcR8crOcg1GZl4XEbdRfrbOyMybACLix5QBxC+g3C3vQsplgp2bCL2D8lcf51H+v3uvs9c0sTtAk2puXPEZ4NOZ+csNrPNXwOcpf03wPEpz3kjchCYiFlG6Ax45wbKDKIM3n5eZ3xt4cQPUDE59RGZOaQBTG5oP93Mol5tdQWnpODMzb9nINo+jBNSPZP/vQ68paAZOPyMzT+tcp5+ZM+HWziPHECBJUqW8T4AkSZUyBEiSVClDgCRJlTIESJJUKUOAJEmVMgRIklSp/w9xwOs/DDENsQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "model.plot_attention(text, attention_matrices, layer_num=2, head_num=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
